/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package com.seari.main;

import com.seari.bean.LogBean;
import com.seari.bean.PageFetcher;
import com.seari.bean.SubSite;
import com.seari.commonutils.common.MiscTools;
import com.seari.commonutils.redis.JedisPoolUtil;
import com.seari.commonutils.redis.JedisUtilProxy;
import com.seari.engine.RollPageSpider;
import com.seari.engine.task.SinglePageTask;
import com.seari.main.listener.VspiderListener;
import com.seari.service.LogService;
import com.seari.service.SubSiteService;
import com.seari.utils.SpiderHelper;
import com.seari.utils.SpringContainer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;

/**
 *
 * @author Rainbow
 */
@Component
@Order(value = 1)
public class Vspider implements CommandLineRunner,VspiderListener
{
    final static String CONFIG_FILE = "config.cfg";
    static Properties config;
    int spiderMode;
    public CopyOnWriteArrayList<RollPageSpider> activeSpiders = new CopyOnWriteArrayList<RollPageSpider>();
    public static final BlockingQueue<PageFetcher> queue = new LinkedBlockingQueue<PageFetcher>(1500);
    public static ExecutorService service = Executors.newCachedThreadPool(); 
    public static AtomicInteger singlePageTaskCount = new AtomicInteger();
    private List<SubSite> subSites = new ArrayList<>();
    private static final Logger LOGGER = LoggerFactory.getLogger(Vspider.class);
    
    @Autowired
    LogService logService;
    
    @Autowired
    private SubSiteService subSiteService;
    
    @Override
    public void run(String... strings) throws Exception
    {
        LOGGER.info("==============run startup program================");
        JedisUtilProxy.initJedisPoolUtil("127.0.0.1", 6379);
        init();
    }

    private void init()
    {
        //初始化
        startSinglePageTask();
        LogBean logBean = new LogBean();
        logBean.setUserName("vpsier");
        logBean.setContent("start up");
        logBean.setType("system");
        logService.addLog(logBean);
        //初始化配置文件
        initConfig();
        if(spiderMode == SpiderHelper.MODE_TEST)
        {
            LOGGER.info("spider run in test mode>>>>>>>>>>");
            subSites = getTestSubsites();
        }else if(spiderMode == SpiderHelper.MODE_NORMAL)
        {
            LOGGER.info("spider run in normal mode>>>>>>>>>>");
            subSites = subSiteService.getSubsites();
        }
        startSpiders();
    }
    
    private void initConfig()
    {
        try
        {
            config = MiscTools.loadConfig(CONFIG_FILE);
            spiderMode = Integer.parseInt(config.getProperty("spider_mode","2"));
        } catch (Exception ex)
        {
            LOGGER.warn(ex.getMessage());
        }
    }
    
    private List<SubSite> getTestSubsites()
    {
        List<SubSite> result = new ArrayList<>();
        //subsite手动初始化
        SubSite subSite = new SubSite();
        subSite.setTableName("bulletin");
        subSite.setClassSpider("CebpubserviceSpider");
        subSite.setTitleContainWords("综合监控");
        subSite.setPageCount(400);
        subSite.setJsEnabled(true);
        subSite.setSiteUrl("http://bulletin.cebpubservice.com/xxfbcmses/search/bulletin.html?searchDate=1994-11-29&dates=300&word=%25E7%25BB%25BC%25E5%2590%2588%25E7%259B%2591%25E6%258E%25A7&categoryId=88&industryName=&area=&status=&publishMedia=&sourceInfo=&showStatus=&startcheckDate={start}&endcheckDate={end}");
        subSite.setTitleLineXpath("//table//tbody//tr");
        subSite.setxLptitleXpath("td//a");
        subSite.setUrlTrait("http.+.(html|htm|php|jsp|asp|aspx)");
        //subSite.setxLPathNextPage("//div[@class=\"pagination\"]//a[contains(string(),\"下一页\")]");
        result.add(subSite);
        return result;
    }
    
    private void startSpiders()
    {
        while (!subSites.isEmpty())
        {            
            SubSite subSite = subSites.remove(0);
            try
            {
                RollPageSpider rollPageSpider;
                if(StringUtils.isEmpty(subSite.getClassSpider()))
                {
                    rollPageSpider = SpringContainer.getBean(RollPageSpider.class);
                }else{
                    rollPageSpider = (RollPageSpider) SpringContainer.getBean(subSite.getClassSpider(), RollPageSpider.class);
                }
                rollPageSpider.initRollPageSpider(subSite);
                new Thread(rollPageSpider).start();
            } catch (Exception e)
            {
                e.printStackTrace();
            }
            
        }
    }
    
    private void startSinglePageTask()
    {
        for(int i = 0;i < 5;i++)
        {
            service.submit(new SinglePageTask(queue));
            singlePageTaskCount.incrementAndGet();
        }
    }

    @Override
    public void spiderStarted(RollPageSpider spider)
    {
        activeSpiders.add(spider);
        LOGGER.info("{} Spider started. Current active spider is {}.", spider,activeSpiders.size());
    }

    @Override
    public void spiderCompleted(RollPageSpider spider)
    {
        activeSpiders.remove(spider);
        LOGGER.info("Engine [{}] completed. Current active spider count is {}.", spider, activeSpiders.size());
    }

    @Override
    public void spiderError(RollPageSpider spider, Throwable ex)
    {
       LOGGER.error("Engine [{}] got problem! Please check.", spider, ex);
    }
    
     public int getActiveSpiderCount()
    {
        return activeSpiders.size();
    }
    
}
